################################################################################-
# Replication File for Wratil, Wäckerle and Proksch: Government Rhetoric and the 
# Representation of Public Opinion in International Negotiations
#
# This script prepares the voting analysis. 
#
# No tables or figures are created in this script.
#
# The CMP dataset cannot be copied to the Dataverse (due to terms of use) and is therefore
# not included in the replication data. In order to recreate the data, please
# go to https://wratil.eu/data and rerun his script to create the file 
# 'parties_cmp2019_online_2.2.dta' and include it in the data folder.
# Unfortunately, this means that this script will not run without this step.
# However, you can just skip this script and go directly to Script #8 and
# work with our pre-loaded dataset.
################################################################################-

library(tidyverse) #version 1.3.2
library(quanteda)  #version 3.2.1
library(readxl)    #version 1.4.0

load(file="generated_data/corpus_final.RData")
docvars.save <- docvars(council_all_nopres)

# Take: Unique Interinstitutional File Number and Actor

votes_1 <- docvars.save %>% 
  select(Actor,Vote1,Vote1Date,Vote1Rule,Interinstitutional_Code1,Council_Config_final,Transcription) %>% 
  dplyr::rename("Vote" = "Vote1",
                "VoteRule" = "Vote1Rule",
                "VoteDate" = "Vote1Date",
                "Interinstitutional_Code" = "Interinstitutional_Code1") %>% 
  filter(!is.na(Vote))
votes_2 <- docvars.save %>% 
  select(Actor,Vote2,Vote2Date,Vote2Rule,Interinstitutional_Code2,Council_Config_final,Transcription) %>% 
  dplyr::rename("Vote" = "Vote2",
                "VoteRule" = "Vote2Rule",
                "VoteDate" = "Vote2Date",
                "Interinstitutional_Code" = "Interinstitutional_Code2") %>% 
  filter(!is.na(Vote))
votes_3 <- docvars.save %>% 
  select(Actor,Vote3,Vote3Date,Vote3Rule,Interinstitutional_Code3,Council_Config_final,Transcription) %>% 
  dplyr::rename("Vote" = "Vote3",
                "VoteRule" = "Vote3Rule",
                "VoteDate" = "Vote3Date",
                "Interinstitutional_Code" = "Interinstitutional_Code3") %>% 
  filter(!is.na(Vote))
votes_4 <- docvars.save %>% 
  select(Actor,Vote4,Vote4Date,Vote4Rule,Interinstitutional_Code4,Council_Config_final,Transcription) %>% 
  dplyr::rename("Vote" = "Vote4",
                "VoteRule" = "Vote4Rule",
                "VoteDate" = "Vote4Date",
                "Interinstitutional_Code" = "Interinstitutional_Code4") %>% 
  filter(!is.na(Vote))
votes_5 <- docvars.save %>% 
  select(Actor,Vote5,Vote5Date,Vote5Rule,Interinstitutional_Code5,Council_Config_final,Transcription) %>% 
  dplyr::rename("Vote" = "Vote5",
                "VoteRule" = "Vote5Rule",
                "VoteDate" = "Vote5Date",
                "Interinstitutional_Code" = "Interinstitutional_Code5") %>% 
  filter(!is.na(Vote))
votes_6 <- docvars.save %>% 
  select(Actor,Vote6,Vote6Date,Vote6Rule,Interinstitutional_Code6,Council_Config_final,Transcription) %>% 
  dplyr::rename("Vote" = "Vote6",
                "VoteRule" = "Vote6Rule",
                "VoteDate" = "Vote6Date",
                "Interinstitutional_Code" = "Interinstitutional_Code6") %>% 
  filter(!is.na(Vote))
votes_7 <- docvars.save %>% 
  select(Actor,Vote7,Vote7Date,Vote7Rule,Interinstitutional_Code7,Council_Config_final,Transcription) %>% 
  dplyr::rename("Vote" = "Vote7",
                "VoteRule" = "Vote7Rule",
                "VoteDate" = "Vote7Date",
                "Interinstitutional_Code" = "Interinstitutional_Code7") %>% 
  filter(!is.na(Vote))
votes_8 <- docvars.save %>% 
  select(Actor,Vote8,Vote8Date,Vote8Rule,Interinstitutional_Code8,Council_Config_final,Transcription) %>% 
  dplyr::rename("Vote" = "Vote8",
                "VoteRule" = "Vote8Rule",
                "VoteDate" = "Vote8Date",
                "Interinstitutional_Code" = "Interinstitutional_Code8") %>% 
  filter(!is.na(Vote))
votes_9 <- docvars.save %>% 
  select(Actor,Vote9,Vote9Date,Vote9Rule,Interinstitutional_Code9,Council_Config_final,Transcription) %>% 
  dplyr::rename("Vote" = "Vote9",
                "VoteRule" = "Vote9Rule",
                "VoteDate" = "Vote9Date",
                "Interinstitutional_Code" = "Interinstitutional_Code9") %>% 
  filter(!is.na(Vote))
votes_10 <- docvars.save %>% 
  select(Actor,Vote10,Vote10Date,Vote10Rule,Interinstitutional_Code10,Council_Config_final,Transcription) %>% 
  dplyr::rename("Vote" = "Vote10",
                "VoteRule" = "Vote10Rule",
                "VoteDate" = "Vote10Date",
                "Interinstitutional_Code" = "Interinstitutional_Code10") %>% 
  filter(!is.na(Vote))
votes_11 <- docvars.save %>% 
  select(Actor,Vote11,Vote11Date,Vote11Rule,Interinstitutional_Code11,Council_Config_final,Transcription) %>% 
  dplyr::rename("Vote" = "Vote11",
                "VoteRule" = "Vote11Rule",
                "VoteDate" = "Vote11Date",
                "Interinstitutional_Code" = "Interinstitutional_Code11") %>% 
  filter(!is.na(Vote))
votes_12 <- docvars.save %>% 
  select(Actor,Vote12,Vote12Date,Vote12Rule,Interinstitutional_Code12,Council_Config_final,Transcription) %>% 
  dplyr::rename("Vote" = "Vote12",
                "VoteRule" = "Vote12Rule",
                "VoteDate" = "Vote12Date",
                "Interinstitutional_Code" = "Interinstitutional_Code12") %>% 
  filter(!is.na(Vote))
votes_13 <- docvars.save %>% 
  select(Actor,Vote13,Vote13Date,Vote13Rule,Interinstitutional_Code13,Council_Config_final,Transcription) %>% 
  dplyr::rename("Vote" = "Vote13",
                "VoteRule" = "Vote13Rule",
                "VoteDate" = "Vote13Date",
                "Interinstitutional_Code" = "Interinstitutional_Code13") %>% 
  filter(!is.na(Vote))
votes_14 <- docvars.save %>% 
  select(Actor,Vote14,Vote14Date,Vote14Rule,Interinstitutional_Code14,Council_Config_final,Transcription) %>% 
  dplyr::rename("Vote" = "Vote14",
                "VoteRule" = "Vote14Rule",
                "VoteDate" = "Vote14Date",
                "Interinstitutional_Code" = "Interinstitutional_Code14") %>% 
  filter(!is.na(Vote))
votes_15 <- docvars.save %>% 
  select(Actor,Vote15,Vote15Date,Vote15Rule,Interinstitutional_Code15,Council_Config_final,Transcription) %>% 
  dplyr::rename("Vote" = "Vote15",
                "VoteRule" = "Vote15Rule",
                "VoteDate" = "Vote15Date",
                "Interinstitutional_Code" = "Interinstitutional_Code15") %>% 
  filter(!is.na(Vote))

all_votes <- bind_rows(votes_1,votes_2,votes_3,votes_4,
                       votes_5,votes_6,votes_7,votes_8,
                       votes_9,votes_10,votes_11,votes_12,
                       votes_13,votes_14,votes_15)
all_votes$code_actor <- paste0(all_votes$Interinstitutional_Code,"_",all_votes$Actor)

configs <- all_votes %>% 
  select(Interinstitutional_Code) %>% 
  distinct() %>% 
  mutate(Council_Config_final = NA)

for(i in 1:nrow(configs)){
  councils <- all_votes %>% 
    filter(Interinstitutional_Code == configs$Interinstitutional_Code[i]) %>% 
    select(Council_Config_final) 
  
  if(n_distinct(councils$Council_Config_final)==1){
    configs$Council_Config_final[i] <- councils$Council_Config_final[1]
  }
  if(n_distinct(councils$Council_Config_final)>1){
    speech_shares <- councils %>% 
      group_by(Council_Config_final) %>% 
      summarise(speeches = n())
    configs$Council_Config_final[i] <- speech_shares$Council_Config_final[which.max(speech_shares$speeches)]
  }
}

all_votes <- all_votes %>% 
  distinct(code_actor, .keep_all = TRUE)

all_votes <- all_votes %>% 
  select(-Council_Config_final) %>% 
  left_join(configs)

table(all_votes$Council_Config_final)

all_votes$date_correct=as.character(all_votes$VoteDate)
all_votes$date_correct=as.Date(all_votes$date_correct,"%d/%m/%Y")

all_votes$year=format(all_votes$date_correct,"%Y")

all_votes <- all_votes %>% 
  filter(!is.na(date_correct))
table(all_votes$Vote,exclude=NULL)

all_votes$VoteRule <- all_votes$VoteRule %>% 
  recode("QMV" = "Qualified majority")

all_votes <- all_votes %>% 
  mutate(Vote_num = ifelse(Vote %in% c("Abstain","No"),1,0))

# Add in Eurobarometer Data
eurobarometer <- readstata13::read.dta13("data/eb_responsiveness_apsr.dta")
eurobarometer$nation <- recode(eurobarometer$nation,
                               "bu" = "bg")

# Add in Manifesto Project Data
cmp_positions <- readstata13::read.dta13("data/parties_cmp2019_online_2.2.dta")
cmp_positions <- cmp_positions %>%
  mutate(actor = case_when(
    country == 1 ~ "BE",
    country == 2 ~ "BG",
    country == 3 ~ "CZ",
    country == 4 ~ "DK",
    country == 5 ~ "DE",
    country == 6 ~ "EE",
    country == 7 ~ "IE",
    country == 8 ~ "EL",
    country == 9 ~ "ES",
    country == 10 ~ "FR",
    country == 11 ~ "HR",
    country == 12 ~ "IT",
    country == 13 ~ "CY",
    country == 14 ~ "LV",
    country == 15 ~ "LT",
    country == 16 ~ "LU",
    country == 17 ~ "HU",
    country == 18 ~ "MT",
    country == 19 ~ "NL",
    country == 20 ~ "AT",
    country == 21 ~ "PL",
    country == 22 ~ "PT",
    country == 23 ~ "RO",
    country == 24 ~ "SI",
    country == 25 ~ "SK",
    country == 26 ~ "FI",
    country == 27 ~ "SE",
    country == 28 ~ "UK"
  ))

eu_budget <- read_xlsx("data/eu_budget_contributions_2016.xlsx")
inflation <- read_xls("data/eurostat_inflation_hicp_2019.xls",sheet = "Ready to load")
unemployment <- read_xls("data/eurostat_unemployment_rate_2019.xls",sheet = "Ready to load")

#-------------------------------
# Join in CMP positions
all_votes$actor_date <- paste(all_votes$Actor,
                              all_votes$date_correct,sep="_")
cmp_positions$actor_date <- paste(cmp_positions$actor,
                                  cmp_positions$date,sep="_")
all_votes <- left_join(all_votes,cmp_positions %>% 
                         select(-"country",-"actor",-"date"),by="actor_date")

#-------------------------------
# Load CHES data
ches_dat <- read_xlsx("data/Cabinet - complete CHES FINAL.xlsx")
ches_join_dat <- data.frame(cabinet_name = unique(ches_dat$cabinet_name),
                            ches_eu_position = NA)

ches_dat$EU_position <- ches_dat$EU_position %>% 
  recode("strongly in favor" = "7")
ches_dat$EU_position <- as.numeric(ches_dat$EU_position)
for(i in 1:nrow(ches_join_dat)){
  ches_dat_temp <- ches_dat %>% 
    filter(cabinet_name==ches_join_dat$cabinet_name[i])
  if(any(is.na(ches_dat_temp$EU_position))){
    cutoff <- 0.8*sum(ches_dat_temp$seats)
    ches_dat_temp_2 <- ches_dat_temp %>% 
      filter(!is.na(EU_position))
    if(sum(ches_dat_temp_2$seats)>cutoff){
      ches_dat_temp <- ches_dat_temp_2
    }
  }
  ches_join_dat$ches_eu_position[i] <- weighted.mean(x = ches_dat_temp$EU_position,
                                                     w = ches_dat_temp$seats)
}

all_votes <- all_votes %>% 
  left_join(ches_join_dat)


#-------------------------------
# Join in Eurobarometer data
eurobarometer$actor_date <- paste(toupper(eurobarometer$nation),
                                  eurobarometer$start_date,sep="_")
all_votes <- left_join(all_votes,eurobarometer %>% 
                         select(-"nation",-"country",-"start_date",-"year"),by="actor_date")

#-------------------------------
# Join in Unemployment data
all_votes$actor_year <- paste(all_votes$Actor,all_votes$year,sep="_")
names(unemployment)[1] <- "government"
unemployment$government=toupper(unemployment$government)
unemployment$actor_year <- paste(unemployment$government,
                                 unemployment$year,sep="_")
all_votes <- left_join(all_votes,unemployment %>%
                         select(-"government",-"year"),by="actor_year")

#-------------------------------
# Join in Inflation data
names(inflation)[1] <- "government"
inflation$government=toupper(inflation$government)
inflation$actor_year <- paste(inflation$government,
                              inflation$year,sep="_")
all_votes <- left_join(all_votes,inflation %>%
                         select(-"government",-"year"),by="actor_year")

#-------------------------------
# Join in EU budget data
names(eu_budget)[1] <- "year"
eu_budget$government=toupper(eu_budget$government)
eu_budget$actor_year <- paste(eu_budget$government,eu_budget$year,sep="_")
all_votes <- left_join(all_votes,eu_budget %>%
                         select(-"government",-"year"),by="actor_year")


all_votes <- all_votes %>% 
  mutate(presidency = case_when(
    date_correct>"2010-07-01" & date_correct<"2010-12-31" ~ "BE",
    date_correct>"2011-01-01" & date_correct<"2011-06-30" ~ "HU",
    date_correct>"2011-07-01" & date_correct<"2011-12-31" ~ "PL",
    date_correct>"2012-01-01" & date_correct<"2012-06-30" ~ "DK",
    date_correct>"2012-07-01" & date_correct<"2012-12-31" ~ "CY",
    date_correct>"2013-01-01" & date_correct<"2013-06-30" ~ "IE",
    date_correct>"2013-07-01" & date_correct<"2013-12-31" ~ "LT",
    date_correct>"2014-01-01" & date_correct<"2014-06-30" ~ "EL",
    date_correct>"2014-07-01" & date_correct<"2014-12-31" ~ "IT",
    date_correct>"2015-01-01" & date_correct<"2015-06-30" ~ "LV",
    date_correct>"2015-07-01" & date_correct<"2015-12-31" ~ "LU",
    date_correct>"2016-01-01" & date_correct<"2016-06-30" ~ "NL",
    date_correct>"2016-07-01" & date_correct<"2016-12-31" ~ "SK",
    date_correct>"2017-01-01" & date_correct<"2017-06-30" ~ "MT",
    date_correct>"2017-07-01" & date_correct<"2017-12-31" ~ "EE",
    date_correct>"2018-01-01" & date_correct<"2018-06-30" ~ "BG"
  ))

all_votes$presidency_vote <- ifelse(all_votes$presidency==all_votes$Actor,"Yes","No")
table(all_votes$presidency_vote)

all_votes$gov_eu_supporter <- ifelse(all_votes$gov_eu_cmp_static>0,"Europhile Government","Eurosceptic Government")
all_votes$gov_eu_supporter_ches <- ifelse(all_votes$ches_eu_position>=4.5,
                                          "Europhile Government",
                                          "Eurosceptic Government")

all_votes$year <- all_votes$year %>% 
  as.factor() %>% 
  relevel("2013")

all_votes <- all_votes %>% 
  mutate(ches_cmp_comb = case_when(
    gov_eu_supporter_ches == "Europhile Government" & gov_eu_supporter == "Europhile Government" ~ "Europhile Government",
    gov_eu_supporter_ches == "Europhile Government" & gov_eu_supporter == "Eurosceptic Government" ~ "Eurosceptic Government",
    gov_eu_supporter_ches == "Eurosceptic Government" & gov_eu_supporter == "Europhile Government" ~ "Eurosceptic Government",
    gov_eu_supporter_ches == "Eurosceptic Government" & gov_eu_supporter == "Eurosceptic Government" ~ "Eurosceptic Government",
    gov_eu_supporter_ches == "Eurosceptic Government" & is.na(gov_eu_supporter) ~ "Eurosceptic Government",
    gov_eu_supporter == "Eurosceptic Government" & is.na(gov_eu_supporter_ches) ~ "Eurosceptic Government",
    gov_eu_supporter_ches == "Europhile Government" & is.na(gov_eu_supporter) ~ "Europhile Government",
    gov_eu_supporter == "Europhile Government" & is.na(gov_eu_supporter_ches) ~ "Europhile Government"
  ))

# save votes -  this only works on Jens' Mac
save(all_votes,file="generated_data/all_votes.RData")
